#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-11-27 16:15:26
# Project: woiwoj_ty_rsf

import re
import sys

from pyspider.libs.base_handler import *
# from pyspider.libs.header_switch import HeadersSelector
from spidermanager.header_switch import HeadersSelector

defaultencoding = 'utf-8'
if sys.getdefaultencoding() != defaultencoding:
    reload(sys)
    sys.setdefaultencoding(defaultencoding)

header_5i5j = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,pl;q=0.7",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "ty.5i5j.com",
    "Referer": "https://ty.5i5j.com/xiaoqu/",
    "Upgrade-Insecure-Requests": 1,
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
    "Cookie": "domain=ty; expires=Tue, 27-Nov-2018 09:51:23 GMT; Max-Age=3600; path=/; domain=5i5j.com",
}


class Handler(BaseHandler):
    crawl_config = {
        "user_agent": "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36",
        "timeout": 120,
        "connect_timeout": 60,
        "retries": 5,
        "fetch_type": 'js',
        "auto_recrawl": True,
    }

    @every(minutes=24 * 60)
    def on_start(self):
        header_slt = HeadersSelector()
        header = header_slt.select_header()
        self.crawl('https://ty.5i5j.com/xiaoqu/', callback=self.index_page, validate_cert=False, headers=header, fetch_type='js')

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('.listTit > a').items():
            self.crawl(each.attr.href, callback=self.detail_page, validate_cert=False, fetch_type='js', headers=header_5i5j)
        self.crawl([x.attr.href for x in response.doc('.cPage').items()], callback=self.index_page,
                   validate_cert=False, fetch_type='js', headers=header_5i5j)

    @config(priority=2)
    def detail_page(self, response):
        return {
            "url": response.url,
            "title": response.doc('.house-tit').text(),
            "price": response.doc('.xqjprice > span').text(),
            "xqfans": response.doc('.xqfangs > li').items(),
            "xqoverview": response.doc('.fl > .clear').items(),
        }
